#coding:utf-8
'''
* coder  : dzlua
* email  : 505544956@qq.com
* module : spider
* path   : whnzhdqs
* file   : whnzhdqs.py
* time   : 2017-11-13 13:59:17
'''
#--------------------#
import sys, codecs, os, time, re
from bs4 import BeautifulSoup
#----------#
sys.path.append("..")
#----------#
from spider import Spider
from database import DataBase
#--------------------#

#--------------------#
class WhnzHdqs(Spider, DataBase):
    error = False
    hrefs = []
    def is_err(self, href):
        if not self.error:
            return True
        for hf in self.hrefs:
            if hf == href:
                return True
            if hf + '.html' == href:
                return True
        return False
    def handle_err(self):
        f = codecs.open(self.__class__.__name__.lower() + '.log', 'r', 'utf-8')
        errs = f.read().split('\n')
        f.close()
        for err in errs:
            b = err.find('http://')
            e = err.find('.html', b)
            self.hrefs.append(err[b:e])
    def __init__(self, host, user, pwd, db, threads_work = 4, tasks = 0):
        DataBase.__init__(self, host, user, pwd, db)
        Spider.__init__(self, threads_work, tasks)

        sql = "CREATE TABLE IF NOT EXISTS " + self.tb + \
            " (id INT AUTO_INCREMENT PRIMARY KEY," \
            " zj TEXT," \
            " title TEXT," \
            " href TEXT," \
            " info TEXT )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)
        #

        self.host = 'http://www.qbxsw.com/'
    def path(self, path):
        return os.path.join(self.host, path)
    def start(self, error=False):
        self.error = error
        self.handle_err()
        self.put(
            self.__on_page1,
            self.path('txt/91/91626') )
    def __on_page1(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find('ul', class_ = ['am-avg-sm-3'])
        if not div:
            self.log('__on_page1, no ul am-avg-sm-3 %s' % url)
            return
        #
        for ele in div.children:
            if ele.name != 'li':
                continue
            a = ele.find('a')
            if not a:
                continue
            #
            href = self.path(a.get('href')[1:])
            txts = a.get('title').split(' ', 1)
            if len(txts) != 2:
                continue
            #
            info = {
                'zj': txts[0],
                'title': txts[1],
                'href': href,
            }
            if self.is_err(href):
                self.put(self.__on_page2, href, None, info)
    def __on_page2(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find('pre', class_ = [ 'am-padding-lg', 'am-text-lg', 'pre-style' ])
        if not div:
            self.log('__on_page2, no pre am-padding-lg am-text-lg pre-style %s' % url)
            return
        #
        text = ''
        for string in div.strings:
            text += (string + '\n')
        #
        args['info'] = text

        self.save(self.__on_save, None, args)

    def __on_save(self, conn, data, args):
        print args['zj'],args['title'],args['href']

        sql = "INSERT INTO " + self.tb + \
            " (zj, title, href, info)" \
            " VALUES(%s,%s,%s,%s);"
        param = ( args['zj'],args['title'],args['href'],args['info'] )
        try:
            cursor = conn.cursor()
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            self.log('__on_save error. %d %s, %s ' % (e[0], e[1], args))
    #--------------------#
    def trans_to_int(self, tt):
        try:
            return int(tt)
        except:
            pass
        tv = 0
        for ch in tt:
            if ch == '0' or ch == u'零':
                tv = tv *10 + 0
            elif ch == u'一' or ch == u'壹' or ch == '1':
                tv = tv *10 + 1
            elif ch == u'二' or ch == '2' or ch == u'贰':
                tv = tv *10 + 2
            elif ch == u'三' or ch == '3':
                tv = tv *10 + 3
            elif ch == u'四' or ch == '4':
                tv = tv *10 + 4
            elif ch == u'五' or ch == '5':
                tv = tv *10 + 5
            elif ch == u'六' or ch == '6':
                tv = tv *10 + 6
            elif ch == u'七' or ch == '7':
                tv = tv *10 + 7
            elif ch == u'八' or ch == '8':
                tv = tv *10 + 8
            elif ch == u'九' or ch == '9':
                tv = tv *10 + 9
            elif ch == u'十' or ch == u'拾':
                pass

        return tv
    def handle(self):
        sql = 'SELECT * FROM %s;' % (self.tb)
        for r in self.select(sql):
            v = self.trans_to_int(r[1])
            print r[1], v
            continue
            if v == 10000:
                continue
            sql = 'UPDATE ' + self.tb + " SET zj=%s WHERE id=%s;"
            param = (v, r[0],)
            self.execute(sql, param)
    def create_file(self, file, content=u''):
        if os.path.exists(file):
            return False
        path = os.path.dirname(file)
        self.mkdirs(path)
        #
        fp = codecs.open(file, 'w', 'utf-8')
        fp.write(content)
        fp.close()

        return True
    def mkdirs(self, path):
        try:
            os.makedirs(path)
        except:
            pass
#--------------------#
